{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/weiqiang/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n",
    "import gc\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from random import choice, seed, shuffle, random, sample\n",
    "import tensorflow as tf\n",
    "import keras.backend.tensorflow_backend as KTF\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, CuDNNGRU as GRU, CuDNNLSTM as LSTM, Dropout, BatchNormalization\n",
    "from keras.layers import Dense, Concatenate, Activation, Embedding, SpatialDropout1D, Bidirectional, Lambda, Conv1D\n",
    "from keras.layers import Add, Average, TimeDistributed, GlobalMaxPooling1D\n",
    "from keras.optimizers import Nadam, Adam, Adamax\n",
    "from keras.activations import absolute_import\n",
    "from keras.legacy import interfaces\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.callbacks import Callback\n",
    "from keras.utils import to_categorical\n",
    "import keras.backend as K\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import keras\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.initializers import he_normal\n",
    "# from keras_bert.bert import get_model\n",
    "from keras_bert.loader import load_trained_model_from_checkpoint\n",
    "from keras_bert import AdamWarmup, calc_train_steps\n",
    "from keras.engine import Layer\n",
    "from keras.engine import InputSpec\n",
    "from keras.objectives import categorical_crossentropy\n",
    "from keras.objectives import sparse_categorical_crossentropy\n",
    "from keras import activations, initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from tqdm import tqdm\n",
    "from model_utils import seq_gather, seq_and_vec, seq_maxpool\n",
    "from keras.models import load_model\n",
    "from keras_bert import get_custom_objects\n",
    "from keras_bert import Tokenizer\n",
    "from collections import defaultdict\n",
    "from eval import read_submission, get_ndcg\n",
    "from tqdm import tqdm, trange\n",
    "import pickle\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True #不全部占满显存, 按需分配\n",
    "sess = tf.Session(config=config)\n",
    "KTF.set_session(sess)\n",
    "\n",
    "BERT_PRETRAINED_DIR = \"../data/uncased_L-12_H-768_A-12/\"\n",
    "VAL_ANS_PATH = '../data/valid_answer.json'\n",
    "LABEL_PATH = '../data/multimodal_labels.txt'\n",
    "\n",
    "MAX_EPOCH = 20\n",
    "MAX_LEN = 20\n",
    "B_SIZE = 128\n",
    "FOLD_IDS = [-1]\n",
    "FOLD_NUM = 20\n",
    "THRE = 0.5\n",
    "SHUFFLE = True\n",
    "MAX_BOX = 10\n",
    "MAX_CHAR = 8\n",
    "PREFIX = \"[image-bert-concat-query]-wwm_uncased_L12-768_v3_quart\"\n",
    "SEED = 2020\n",
    "ACCUM_STEP = int(128 // B_SIZE)\n",
    "SAVE_EPOCHS=[10, 20, 35, 50, 80, 100]\n",
    "IMAGE_LABEM_CONCAT_TOKEN = \"###\"\n",
    "CONCAT_TOKE = \"[unused0]\"\n",
    "\n",
    "cfg = {}\n",
    "cfg[\"verbose\"] = PREFIX\n",
    "cfg[\"base_dir\"] = BERT_PRETRAINED_DIR\n",
    "cfg['maxlen'] = MAX_LEN\n",
    "cfg[\"max_box\"] = MAX_BOX\n",
    "cfg[\"max_char\"] = MAX_CHAR\n",
    "cfg[\"lr\"] = 1e-4\n",
    "cfg['min_lr'] = 6e-8\n",
    "cfg[\"opt\"] = \"nadam\"\n",
    "cfg[\"loss_w\"] =  20.\n",
    "cfg[\"trainable\"] = True\n",
    "cfg[\"bert_trainable\"] = True\n",
    "cfg[\"mix_mode\"] = \"\"   # add concat average\n",
    "cfg[\"unit1_1\"] = 128\n",
    "cfg[\"accum_step\"] = ACCUM_STEP\n",
    "cfg[\"cls_num\"] = 4\n",
    "cfg[\"raw_filename\"] = \"{}_{}oof{}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vocab():\n",
    "    if \"albert\"in cfg[\"verbose\"].lower():\n",
    "        dict_path = os.path.join(BERT_PRETRAINED_DIR, 'vocab_chinese.txt')\n",
    "    else:\n",
    "        dict_path = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')\n",
    "    with open(dict_path, mode=\"r\", encoding=\"utf8\") as f:\n",
    "        lines = f.readlines()\n",
    "        lines = [l.strip() for l in lines]\n",
    "\n",
    "    word_index = {v: k  for k, v in enumerate(lines)}\n",
    "    return word_index\n",
    "\n",
    "\n",
    "word_index = get_vocab()\n",
    "cfg[\"x_pad\"] = word_index[\"[PAD]\"]\n",
    "tokenizer = Tokenizer(word_index)\n",
    "\n",
    "\n",
    "def get_label(path):\n",
    "    with open(path) as f:\n",
    "        lines = f.readlines()\n",
    "        label2id = {l.split('\\n')[0].split('\\t')[1]:int(l.split('\\n')[0].split('\\t')[0]) for l in lines[1:]}\n",
    "        id2label = {int(l.split('\\n')[0].split('\\t')[0]):l.split('\\n')[0].split('\\t')[1] for l in lines[1:]}\n",
    "    return label2id, id2label\n",
    "\n",
    "\n",
    "label2id, id2label = get_label(LABEL_PATH)\n",
    "label_set = set(label2id.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000 <class 'pandas.core.frame.DataFrame'>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>label_words</th>\n",
       "      <th>features</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3373</th>\n",
       "      <td>northeast organic black fungus</td>\n",
       "      <td>others###others###others</td>\n",
       "      <td>[[0.0, 0.0, 1.2517451, 0.0, 0.15538946, 0.0, 0...</td>\n",
       "      <td>[[0.1375, 0.4075, 0.08125, 0.50625, 0.11475], ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7382</th>\n",
       "      <td>huan yan silk facial cream</td>\n",
       "      <td>others###others###human face</td>\n",
       "      <td>[[0.0, 0.0, 0.0665189, 0.0, 0.11938095, 0.0, 0...</td>\n",
       "      <td>[[0.3657142857142857, 0.9742857142857143, 0.56...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5047</th>\n",
       "      <td>sleeveless hedging leotard</td>\n",
       "      <td>top clothes (coat, jacket, shirt, etc.)###top ...</td>\n",
       "      <td>[[0.0, 0.0, 0.595602, 0.0, 0.0, 0.0, 0.0285581...</td>\n",
       "      <td>[[0.15875, 0.74375, 0.07625, 0.53625, 0.2691],...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               words  \\\n",
       "3373  northeast organic black fungus   \n",
       "7382      huan yan silk facial cream   \n",
       "5047      sleeveless hedging leotard   \n",
       "\n",
       "                                            label_words  \\\n",
       "3373                           others###others###others   \n",
       "7382                       others###others###human face   \n",
       "5047  top clothes (coat, jacket, shirt, etc.)###top ...   \n",
       "\n",
       "                                               features  \\\n",
       "3373  [[0.0, 0.0, 1.2517451, 0.0, 0.15538946, 0.0, 0...   \n",
       "7382  [[0.0, 0.0, 0.0665189, 0.0, 0.11938095, 0.0, 0...   \n",
       "5047  [[0.0, 0.0, 0.595602, 0.0, 0.0, 0.0, 0.0285581...   \n",
       "\n",
       "                                                    pos  \n",
       "3373  [[0.1375, 0.4075, 0.08125, 0.50625, 0.11475], ...  \n",
       "7382  [[0.3657142857142857, 0.9742857142857143, 0.56...  \n",
       "5047  [[0.15875, 0.74375, 0.07625, 0.53625, 0.2691],...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 全量数据\n",
    "# with open('../data/train_data.pkl', 'rb') as outp:\n",
    "#     train_data= joblib.load(outp)\n",
    "# 1w条\n",
    "with open('../data/temp_data.pkl', 'rb') as outp:\n",
    "    train_data = joblib.load(outp)\n",
    "\n",
    "\n",
    "with open('../data/val_data.pkl', 'rb') as outp:\n",
    "    val_data = pickle.load(outp)\n",
    "    \n",
    "print(len(train_data), type(train_data))\n",
    "train_data.reset_index(drop=True, inplace=True)\n",
    "train_data.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr):\n",
    "    return word, np.asarray(arr, dtype=np.float16)\n",
    "\n",
    "\n",
    "def load_embed(path, dim=300, word_index=None):\n",
    "    embedding_index = {}\n",
    "    with open(path, mode=\"r\", encoding=\"utf8\") as f:\n",
    "        lines = f.readlines()\n",
    "        for l in lines:\n",
    "            l = l.strip().split()\n",
    "            word, arr = l[0], l[1:]\n",
    "            if len(arr) != dim:\n",
    "                print(\"[!] l = {}\".format(l))\n",
    "                continue\n",
    "            if word_index and word not in word_index:\n",
    "                continue\n",
    "            word, arr = get_coefs(word, arr)\n",
    "            embedding_index[word] = arr\n",
    "    return embedding_index\n",
    "\n",
    "\n",
    "def build_matrix(path, word_index=None, max_features=None, dim=300):\n",
    "    embedding_index = load_embed(path, dim=dim, word_index=word_index)\n",
    "    max_features = len(word_index) + 1 if max_features is None else max_features \n",
    "    embedding_matrix = np.zeros((max_features + 1, dim))\n",
    "    unknown_words = []\n",
    "    \n",
    "    for word, i in word_index.items():\n",
    "        if i <= max_features:\n",
    "            try:\n",
    "                embedding_matrix[i] = embedding_index[word]\n",
    "            except KeyError:\n",
    "#                 print(word)\n",
    "                unknown_words.append(word)\n",
    "    return embedding_matrix, unknown_words\n",
    "\n",
    "\n",
    "def load_word_embed(word_embed_glove=\"../data/glove.840B.300d.txt\", \n",
    "                    word_embed_crawl=\"../data/crawl-300d-2M.vec\",\n",
    "               save_filename=\"./word_embedding_matrix\",\n",
    "               word_index=None):\n",
    "    \"\"\"\n",
    "    (30524, 300) 7590\n",
    "    (30524, 300) 7218\n",
    "    \"\"\"    \n",
    "    if os.path.exists(save_filename + \".npy\"):\n",
    "        word_embedding_matrix = np.load(save_filename + \".npy\").astype(\"float32\")\n",
    "    else:\n",
    "        word_embedding_matrix, tx_unk = build_matrix(word_embed_glove, word_index=word_index, dim=300)\n",
    "\n",
    "        print(word_embedding_matrix.shape, len(tx_unk))\n",
    "        \n",
    "        word_embedding_matrix_v2, tx_unk = build_matrix(word_embed_crawl, word_index=word_index, dim=300)\n",
    "\n",
    "        print(word_embedding_matrix_v2.shape, len(tx_unk))\n",
    "        \n",
    "        word_embedding_matrix = np.concatenate([word_embedding_matrix, word_embedding_matrix_v2], axis=1)\n",
    "        \n",
    "        gc.collect()\n",
    "        np.save(save_filename, word_embedding_matrix)\n",
    "    return word_embedding_matrix\n",
    "\n",
    "\n",
    "word_embedding_matrix = load_word_embed(word_index=word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(cfg, summary=False, word_embedding_matrix=None):\n",
    "    \n",
    "    def _get_model(base_dir, cfg_=None):\n",
    "        config_file = os.path.join(base_dir, 'bert_config.json')\n",
    "        checkpoint_file = os.path.join(base_dir, 'bert_model.ckpt')\n",
    "        if not os.path.exists(config_file):\n",
    "            config_file = os.path.join(base_dir, 'bert_config_large.json')\n",
    "            checkpoint_file = os.path.join(base_dir, 'roberta_l24_large_model')\n",
    "        print(config_file, checkpoint_file)\n",
    "#         model = load_trained_model_from_checkpoint(config_file, checkpoint_file, training=True, seq_len=cfg_['maxlen'])\n",
    "        model = load_trained_model_from_checkpoint(config_file, \n",
    "                                           checkpoint_file, \n",
    "                                           training=False, \n",
    "                                           trainable=cfg_[\"bert_trainable\"], \n",
    "                                           output_layer_num=cfg[\"cls_num\"],\n",
    "                                           seq_len=cfg_['maxlen'])\n",
    "        return model\n",
    "    \n",
    "    def get_opt(num_example, warmup_proportion=0.1, lr=2e-5, min_lr=None):\n",
    "        if cfg[\"opt\"].lower() == \"nadam\":\n",
    "            opt = Nadam(lr=lr)\n",
    "        else:\n",
    "            total_steps, warmup_steps = calc_train_steps(\n",
    "                num_example=num_example,\n",
    "                batch_size=B_SIZE,\n",
    "                epochs=MAX_EPOCH,\n",
    "                warmup_proportion=warmup_proportion,\n",
    "            )\n",
    "\n",
    "            opt = AdamWarmup(total_steps, warmup_steps, lr=lr, min_lr=min_lr)\n",
    "\n",
    "        return opt\n",
    "\n",
    "    model1 = _get_model(cfg[\"base_dir\"], cfg)\n",
    "    model1 = Model(inputs=model1.inputs[: 2], outputs=model1.layers[-7].output)\n",
    "\n",
    "    if word_embedding_matrix is not None:\n",
    "        embed_layer = Embedding(input_dim=word_embedding_matrix.shape[0], \n",
    "                                output_dim=word_embedding_matrix.shape[1],\n",
    "                                weights=[word_embedding_matrix],\n",
    "                                trainable=cfg[\"trainable\"],\n",
    "                                name=\"embed_layer\"\n",
    "                         )\n",
    "        \n",
    "    inp_token1 = Input(shape=(None, ), dtype=np.int32, name=\"query_token_input\")\n",
    "    inp_segm1 = Input(shape=(None, ), dtype=np.float32, name=\"query_segm_input\")\n",
    "    \n",
    "#     inp_token2 = Input(shape=(None, ), dtype=np.int32)\n",
    "#     inp_segm2 = Input(shape=(None, ), dtype=np.float32)    \n",
    "    \n",
    "    inp_image = Input(shape=(None, 2048), dtype=np.float32, name=\"image_input\")\n",
    "    inp_image_mask = Input(shape=(None, ), dtype=np.float32, name=\"image_mask_input\")\n",
    "    inp_pos = Input(shape=(None, 5), dtype=np.float32, name=\"image_pos_input\")        \n",
    "    inp_image_char = Input(shape=(None, cfg[\"max_char\"]), dtype=np.int32, name='image_char_input')\n",
    "    \n",
    "    \n",
    "    mask = Lambda(lambda x: K.cast(K.not_equal(x, cfg[\"x_pad\"]), 'float32'), name=\"token_mask\")(inp_token1)\n",
    "    word_embed = embed_layer(inp_token1)\n",
    "    word_embed = Lambda(lambda x: x[0] * K.expand_dims(x[1], axis=-1))([word_embed, mask])\n",
    "    word_embed = Bidirectional(LSTM(cfg[\"unit1_1\"], return_sequences=True), merge_mode=\"sum\")(word_embed)\n",
    "    word_embed = Lambda(lambda x: x[0] * K.expand_dims(x[1], axis=-1))([word_embed, mask])\n",
    "\n",
    "    sequence_output = model1([inp_token1, inp_segm1])\n",
    "    sequence_output = Concatenate(axis=-1)([sequence_output, word_embed])\n",
    "    text_pool = Lambda(lambda x: x[:, 0, :])(sequence_output)\n",
    "\n",
    "    # Share weights of character-level embedding for premise and hypothesis\n",
    "    character_embedding_layer = TimeDistributed(Sequential([\n",
    "        embed_layer,\n",
    "        # Embedding(input_dim=100, output_dim=char_embedding_size, input_length=chars_per_word),\n",
    "        Conv1D(filters=128, kernel_size=3, name=\"char_embed_conv1d\"),\n",
    "        GlobalMaxPooling1D()\n",
    "    ]), name='CharEmbedding')\n",
    "    character_embedding_layer.build(input_shape=(None, None, cfg[\"max_char\"]))\n",
    "    image_char_embed  = character_embedding_layer(inp_image_char)    \n",
    "    image_embed = Concatenate(axis=-1)([image_char_embed, inp_image])    \n",
    "    image_embed = Dense(512, activation='relu', name='image_embed')(image_embed)\n",
    "    image_embed = Lambda(lambda x: x[0] * K.expand_dims(x[1], axis=-1))([image_embed, inp_image_mask])\n",
    "    pos_embed = Dense(512, activation='relu', name='pos_embed')(inp_pos)\n",
    "    pos_embed = Lambda(lambda x: x[0] * K.expand_dims(x[1], axis=-1))([pos_embed, inp_image_mask])\n",
    "    embed = Add()([image_embed , pos_embed]) # batch, maxlen(10), 1024+128\n",
    "    \n",
    "    image_embed = Bidirectional(LSTM(1152, return_sequences=True), merge_mode=\"sum\")(embed)\n",
    "    image_embed = Lambda(lambda x: x[0] * K.expand_dims(x[1], axis=-1))([image_embed, inp_image_mask])\n",
    "    \n",
    "    image_pool = Lambda(lambda x: x[:, 0, :])(image_embed)\n",
    "    \n",
    "    pool = Concatenate(axis=-1)([image_pool, text_pool])\n",
    "    pool = Dense(2048, activation=\"relu\")(pool)\n",
    "    pool = Dense(512, activation=\"relu\")(pool)\n",
    "    pool = Dense(128, activation=\"relu\")(pool)\n",
    "    \n",
    "    output = Dense(2, activation='softmax', name='output')(pool)\n",
    "\n",
    "    opt = get_opt(num_example=cfg[\"num_example\"], lr=cfg[\"lr\"], min_lr=cfg['min_lr'])\n",
    "    model = Model(inputs=[inp_token1, inp_segm1, \n",
    "                          inp_image, inp_image_mask,\n",
    "                          inp_pos, inp_image_char], outputs=[output])#\n",
    "    \n",
    "    model.compile(optimizer=opt, loss={\n",
    "                'output': 'sparse_categorical_crossentropy'\n",
    "            }, metrics=['accuracy'])\n",
    "    if summary:\n",
    "        model.summary()\n",
    "    \n",
    "    return model\n",
    "\n",
    "# cfg[\"num_example\"] = 32\n",
    "# model = build_model(cfg, summary=True, word_embedding_matrix=word_embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _truncate_seq_pair(tokens_a, tokens_b, max_length):\n",
    "    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n",
    "\n",
    "    # This is a simple heuristic which will always truncate the longer sequence\n",
    "    # one token at a time. This makes more sense than truncating an equal percent\n",
    "    # of tokens from each, since if one sequence is very short then each token\n",
    "    # that's truncated likely contains more information than a longer sequence.\n",
    "\n",
    "    while True:\n",
    "        total_length = len(tokens_a) + len(tokens_b)\n",
    "        if total_length <= max_length:\n",
    "            break\n",
    "        if len(tokens_a) > len(tokens_b):\n",
    "            tokens_a.pop()\n",
    "        else:\n",
    "            print(len(tokens_a))\n",
    "            tokens_b.pop()\n",
    "\n",
    "\n",
    "def token2id_X(X, x_dict, maxlen=None):\n",
    "    x = tokenizer.tokenize(X)\n",
    "    if maxlen:\n",
    "        x = x[: 1] + list(x)[1: maxlen - 1] + x[-1: ]     \n",
    "    seg = [0 for _ in x]\n",
    "    token = list(x)\n",
    "    x = [x_dict[e] if e in x_dict else x_dict[\"[UNK]\"] for e in token]\n",
    "    assert len(x) == len(seg)\n",
    "    return x, seg\n",
    "\n",
    "\n",
    "def seq_padding(X, maxlen=None, padding_value=None, debug=False):\n",
    "    L = [len(x) for x in X]\n",
    "    if maxlen is None:\n",
    "        maxlen = max(L)\n",
    "\n",
    "    pad_X = np.array([\n",
    "        np.concatenate([x, [padding_value] * (maxlen - len(x))]) if len(x) < maxlen else x[: maxlen] for x in X\n",
    "    ])\n",
    "    if debug:\n",
    "        print(\"[!] before pading {}\\n\".format(X))\n",
    "        print(\"[!] after pading {}\\n\".format(pad_X))\n",
    "    return pad_X\n",
    "    \n",
    "\n",
    "def MyChoice(Myset):\n",
    "    result = []\n",
    "    for i in Myset:\n",
    "        temp_set = set()\n",
    "        temp_set.add(i)\n",
    "        cho = choice(list(Myset - temp_set))\n",
    "        result.append(cho)\n",
    "    return result\n",
    "\n",
    "\n",
    "class data_generator:\n",
    "    \n",
    "    def __init__(self, data, batch_size=B_SIZE, shuffle=SHUFFLE):\n",
    "        self.data = data\n",
    "        self.batch_size = batch_size\n",
    "        self.steps = len(self.data) // self.batch_size\n",
    "        self.shuffle = shuffle\n",
    "        if len(self.data) % self.batch_size != 0:\n",
    "            self.steps += 1\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.steps\n",
    "    \n",
    "    def __iter__(self):\n",
    "        \"\"\"\n",
    "        inp_token1,\n",
    "        inp_segm1,\n",
    "        inp_image,\n",
    "        inp_image_mask,\n",
    "        inp_pos, \n",
    "        inp_image_char\n",
    "        \"\"\"\n",
    "        while True:\n",
    "            idxs = list(range(len(self.data)))\n",
    "            if self.shuffle:\n",
    "                np.random.shuffle(idxs)\n",
    "            T1, T2, Image1, Pos1, label_word_list, image1_mask, image1_char = [], [], [], [], [], [], []\n",
    "            S1, S2, Image2, Pos2, image2_mask, image2_char = [], [], [], [], [], [] # 负样本\n",
    "            Id_set = set()\n",
    "\n",
    "            for i in idxs:\n",
    "                d = self.data.iloc[i]\n",
    "                text = d['words']\n",
    "                label_words = d['label_words']\n",
    "                \n",
    "                t1, t2 = token2id_X(text, x_dict=word_index, maxlen=cfg[\"maxlen\"])\n",
    "                image = np.array(d['features'], dtype=\"float32\")\n",
    "                image = image[: cfg[\"max_box\"]]\n",
    "                img_mask = [1 for _ in image[: cfg[\"max_box\"]]]\n",
    "                \n",
    "                pos = np.array(d['pos'], dtype=\"float32\")\n",
    "                pos = pos[: cfg[\"max_box\"]]\n",
    "                \n",
    "                image_char = [token2id_X(ent, x_dict=word_index)[0] for ent in label_words.split(IMAGE_LABEM_CONCAT_TOKEN)]\n",
    "                image_char = image_char[: cfg[\"max_box\"]]\n",
    "                # print(\"image_char\", len(image_char))\n",
    "                image_char = pad_sequences(image_char, \n",
    "                                           maxlen=cfg[\"max_char\"], \n",
    "                                           dtype='int32',\n",
    "                                           padding='post',\n",
    "                                           truncating='post',\n",
    "                                           value=cfg[\"x_pad\"])\n",
    "                \n",
    "                assert image.shape[0] == pos.shape[0]\n",
    "                assert image.shape[0] == cfg[\"max_box\"] or image.shape[0] == len(label_words.split(IMAGE_LABEM_CONCAT_TOKEN))\n",
    "                assert image_char.shape == (image.shape[0], cfg[\"max_char\"])\n",
    "\n",
    "                T1.append(t1)\n",
    "                T2.append(t2)\n",
    "                Image1.append(image)\n",
    "                image1_mask.append(img_mask)  \n",
    "                Pos1.append(pos)\n",
    "                image1_char.append(image_char)\n",
    "                Id_set.add(i)\n",
    "\n",
    "                if len(T1) == self.batch_size//2 or i == idxs[-1]:\n",
    "                    ## 加入负样本\n",
    "                    Id_new = MyChoice(Id_set)\n",
    "#                     print(Id_set, Id_new)\n",
    "                    for i, id_ in enumerate(Id_new):\n",
    "                        d_new = self.data.iloc[id_]\n",
    "                        text = d_new['words']\n",
    "                        t1, t2 = token2id_X(text, x_dict=word_index, maxlen=cfg[\"maxlen\"])\n",
    "                        S1.append(t1)\n",
    "                        S2.append(t2)\n",
    "                        \n",
    "                        image = Image1[i]\n",
    "                        img_mask = image1_mask[i]\n",
    "                        pos = Pos1[i]\n",
    "                        image_char = image1_char[i]\n",
    "                        \n",
    "                        Image2.append(image)\n",
    "                        Pos2.append(pos)\n",
    "                        image2_mask.append(img_mask)\n",
    "                        image2_char.append(image_char)\n",
    "                    \n",
    "                    Y = [1] * len(T1) + [0] * len(S1)\n",
    "                   \n",
    "                    T1 = seq_padding(T1 + S1, padding_value=cfg[\"x_pad\"]) \n",
    "                    T2 = seq_padding(T2 + S2, padding_value=cfg[\"x_pad\"])\n",
    "                    \n",
    "                    Image1 = seq_padding(Image1 + Image2, \n",
    "                                         padding_value=np.zeros(shape=(2048, ))\n",
    "                                        )\n",
    "                                                         \n",
    "                    Pos1 = seq_padding(Pos1 + Pos2,\n",
    "                                       padding_value=np.zeros(shape=(5, ))\n",
    "                                      )\n",
    "                    image1_mask = seq_padding(image1_mask + image2_mask,\n",
    "                                             padding_value=0)\n",
    "                    \n",
    "                    image1_char = seq_padding(image1_char + image2_char,\n",
    "                                             padding_value=np.zeros(shape=(cfg[\"max_char\"])), debug=False)\n",
    "                    \n",
    "                    Y = np.array(Y).reshape((len(T1), -1))\n",
    "                    \n",
    "                    idx = np.arange(len(T1))\n",
    "                    np.random.shuffle(idx)\n",
    "        \n",
    "                    T1 = T1[idx]\n",
    "                    T2 = T2[idx]\n",
    "                    Image1 = Image1[idx]\n",
    "                    image1_mask = image1_mask[idx]\n",
    "                    Pos1 = Pos1[idx]\n",
    "                    image1_char = image1_char[idx]\n",
    "                    Y = Y[idx]\n",
    "                    \n",
    "                    yield [T1, T2, Image1, image1_mask, Pos1, image1_char], Y\n",
    "                    T1, T2, Image1, Pos1, label_word_list, image1_mask, image1_char = [], [], [], [], [], [], []\n",
    "                    S1, S2, Image2, Pos2, image2_mask, image2_char = [], [], [], [], [], [] # 负样本\n",
    "                    Id_set = set()\n",
    "\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [54190. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 11102. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [37880.  2703.  2703. ... 54190.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 63124. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [54190. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.  7593. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 7593.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [33096. 21740.  4959. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 63124. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [33096. 29895.  6210. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  4959.  4959. ...     0.     0.     0.]\n",
      " [37880.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 10) (128, 10) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703. 29895. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [54190.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.  6210.  6210. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[33096.  4959.  4959. ... 29895.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [33096. 29895. 54190. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  6210. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [37880.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 37880.     0. ...     0.     0.     0.]\n",
      " [ 6210.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210. 33096.  6210. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  7593.  7593. ...     0.     0.     0.]\n",
      " ...\n",
      " [11102.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [33096. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [37880. 37880. 37880. ...     0.     0.     0.]\n",
      " [21740. 29895. 54190. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [54190. 21740.     0. ...     0.     0.     0.]\n",
      " [21740. 21740. 21740. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [21740. 33096. 29895. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 15348. 21740. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.     0.     0.]\n",
      " [29895. 21740.  6210. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959. 54190. 21740. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959. 15348. 54190. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 37880.  2703. ...     0.     0.     0.]\n",
      " [ 7593.  7593.  6210. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[15348.  6210.  6210. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 21740. 11102. ...     0.     0.     0.]\n",
      " [ 2703. 63124.  2703. ...     0.     0.     0.]\n",
      " [21740. 33096. 29895. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 4959. 21740. 15348. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 21740.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 11102. ...     0.     0.     0.]\n",
      " [54190. 21740. 29895. ...  6210.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 9887.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 21740. 21740. ...     0.     0.     0.]\n",
      " [ 7593.  2703.  2703. ...  7593.  2703.  2703.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[29895. 29895.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.     0. ...     0.     0.     0.]\n",
      " [29895. 29895. 29895. ...  6210.  4959. 54190.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [37880.  4959. 54190. ...     0.     0.     0.]\n",
      " [ 7593.  7593.  2703. ...  7593.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [11102.     0.     0. ...     0.     0.     0.]\n",
      " [15348.  6210.  4959. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703. 29895. 37880. ...     0.     0.     0.]\n",
      " ...\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " [11102.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 29895.     0. ...     0.     0.     0.]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [63124. 63124. 63124. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[33096. 21740. 54190. ...     0.     0.     0.]\n",
      " [ 2703. 37880. 37880. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.  2703.  2703. ...  2703.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [63124.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740. 54190.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [11102. 11102. 11102. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [11102. 11102. 11102. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  2703. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703. 63124. ...  2703.  2703.  2703.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 4959.  2703. 21740. ...     0.     0.     0.]\n",
      " [ 4959. 21740. 54190. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [33096.  2703.  6210. ...  2703.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  7593.     0. ...     0.     0.     0.]\n",
      " [54190. 54190. 15348. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 21740. 54190. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [54190. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [15348. 54190. 54190. ...     0.     0.     0.]\n",
      " [21740.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 4959.  2703. 11102. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [11102.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [33096.  6210.  6210. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  9887. ...     0.     0.     0.]\n",
      " ...\n",
      " [29895.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 7593.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 11102.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 6210.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 63124. 63124. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[11102.  2703.     0. ...     0.     0.     0.]\n",
      " [33096.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [11102. 11102.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[63124. 63124.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210. 33096. ...     0.     0.     0.]\n",
      " [ 6210.  6210. 33096. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [54190.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703. 15348.     0. ...     0.     0.     0.]\n",
      " [ 6210.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [11102.  6210. 33096. ...     0.     0.     0.]\n",
      " [21740. 33096.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 9, 2048) (128, 9) (128, 9, 5) (128, 9, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  4959.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  4959.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  6210. ...     0.     0.     0.]\n",
      " [37880.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703. 10430. ... 37880.  2703.  2703.]\n",
      " [21740. 54190.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 54190. ...  2703.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[2703.    0.    0. ...    0.    0.    0.]\n",
      " [2703. 2703.    0. ...    0.    0.    0.]\n",
      " [2703. 2703.    0. ...    0.    0.    0.]\n",
      " ...\n",
      " [2703.    0.    0. ...    0.    0.    0.]\n",
      " [2703. 2703. 2703. ...    0.    0.    0.]\n",
      " [2703. 2703.    0. ...    0.    0.    0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [29895. 29895.  2703. ...  2703.  4959.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[37880. 37880.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.  4959. 21740. ...     0.     0.     0.]\n",
      " [ 4959.  2703. 11102. ...     0.     0.     0.]\n",
      " [10430.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 54190.  4959. ...     0.     0.     0.]\n",
      " [ 2703. 63124. 63124. ...     0.     0.     0.]\n",
      " [29895. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.     0.     0. ...     0.     0.     0.]\n",
      " [54190.  6210.  6210. ...     0.     0.     0.]\n",
      " [54190.  4959. 54190. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 54190.  4959. ...     0.     0.     0.]\n",
      " [ 2703. 11102. 10430. ...  2703.  2703.     0.]\n",
      " [54190.  4959. 21740. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [11102. 11102. 11102. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 7593.     0.     0. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  2703. ...     0.     0.     0.]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.     0.]\n",
      " [ 6210.  6210.  6210. ...  6210.     0.     0.]\n",
      " ...\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [21740.  4959. 33096. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [63124. 63124. 54190. ... 63124.     0.     0.]\n",
      " ...\n",
      " [11102.  2703. 11102. ...  2703. 11102.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 29895.  7593. ...  2703.  2703.  2703.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 4959. 21740. 33096. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [33096. 29895. 21740. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [21740.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 15348.  4959. ...     0.     0.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " [33096. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 4959.  2703. 33096. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 63124. ...     0.     0.     0.]\n",
      " [ 4959.  2703. 21740. ...  2703. 54190. 54190.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [37880. 37880.     0. ...     0.     0.     0.]\n",
      " [ 4959. 15348. 54190. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.     0.     0. ...     0.     0.     0.]\n",
      " [11102.  2703. 11102. ... 11102. 11102.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.  2703.  4959. ...  6210. 54190.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [63124. 63124. 63124. ...  2703.  2703. 63124.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703. 63124. 63124. ... 63124. 63124. 63124.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 4959. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703. 11102. 11102. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 37880.  2703. ...  2703.  2703.  2703.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 6210.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190.  4959. 54190. ...     0.     0.     0.]\n",
      " [ 2703. 21740.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [29895. 29895. 29895. ...     0.     0.     0.]\n",
      " [33096. 21740.  4959. ...     0.     0.     0.]\n",
      " [ 7593.  7593.  2703. ...  2703.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[2703.    0.    0. ...    0.    0.    0.]\n",
      " [6210. 2703. 6210. ...    0.    0.    0.]\n",
      " [2703. 2703.    0. ...    0.    0.    0.]\n",
      " ...\n",
      " [2703. 2703. 2703. ...    0.    0.    0.]\n",
      " [6210. 2703. 6210. ...    0.    0.    0.]\n",
      " [2703.    0.    0. ...    0.    0.    0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[29895. 54190. 15348. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [54190.  4959. 54190. ... 54190. 33096. 33096.]\n",
      " [63124.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 11102.     0. ...     0.     0.     0.]\n",
      " [ 2703. 11102. 11102. ... 11102. 11102.     0.]\n",
      " [ 9887.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  4959. ...     0.     0.     0.]\n",
      " [ 7593.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " [21740. 33096. 21740. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [29895.  4959. 21740. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " [ 2703. 37880.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [21740.  4959.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  7593.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 7593.     0.     0. ...     0.     0.     0.]\n",
      " [ 7593.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703. 37880.     0. ...     0.     0.     0.]\n",
      " [ 4959. 21740.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190. 21740.  4959. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 54190.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  4959. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740. 54190. 21740. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [29895. 33096. 33096. ... 33096. 33096.  2703.]]\n",
      "x (128, 10) (128, 10) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [11102.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703. 63124. ...  2703.  2703.  2703.]\n",
      " [ 2703.  2703. 63124. ...  2703.  2703.  2703.]\n",
      " [ 2703. 11102.  2703. ...  2703.  2703. 11102.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959. 21740.  6210. ...  6210.  6210.     0.]\n",
      " [15348. 54190.  2703. ... 15348.     0.     0.]\n",
      " ...\n",
      " [11102. 11102.     0. ...     0.     0.     0.]\n",
      " [15348. 15348.  4959. ...  6210.     0.     0.]\n",
      " [ 4959. 21740.  6210. ...  6210.  6210.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [33096.  4959. 21740. ...     0.     0.     0.]\n",
      " [54190. 54190.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [33096.     0.     0. ...     0.     0.     0.]\n",
      " [54190.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[33096. 33096.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  2703. ...     0.     0.     0.]\n",
      " [ 7593.  2703.  2703. ...  2703.  2703.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 15) (128, 15) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [21740. 33096.  4959. ...     0.     0.     0.]\n",
      " ...\n",
      " [37880.  2703. 37880. ... 37880.  2703. 37880.]\n",
      " [11102.  2703.     0. ...     0.     0.     0.]\n",
      " [21740. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [54190. 33096.  4959. ...     0.     0.     0.]\n",
      " [ 9887.  2703.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [29895.  9887.  2703. ...     0.     0.     0.]\n",
      " [ 2703. 21740. 33096. ...     0.     0.     0.]\n",
      " [29895. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190. 54190. 21740. ...  6210.  6210.  6210.]\n",
      " [21740.  2703.  6210. ...     0.     0.     0.]\n",
      " [ 6210. 33096.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703. 63124.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703. 29895. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  7593.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 7593. 15348.  4959. ...     0.     0.     0.]\n",
      " [ 4959. 21740.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[2703. 2703.    0. ...    0.    0.    0.]\n",
      " [2703. 2703.    0. ...    0.    0.    0.]\n",
      " [2703.    0.    0. ...    0.    0.    0.]\n",
      " ...\n",
      " [6210. 6210.    0. ...    0.    0.    0.]\n",
      " [2703.    0.    0. ...    0.    0.    0.]\n",
      " [2703. 2703. 2703. ...    0.    0.    0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.  2703.  6210. ...     0.     0.     0.]\n",
      " [21740. 54190. 54190. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210. 21740. 29895. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  7593. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703. 11102. ...     0.     0.     0.]\n",
      " [ 6210. 21740. 33096. ...  6210.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.  2703.  2703.]\n",
      " [ 2703. 11102.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [63124. 63124. 63124. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703. 11102. 11102. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[37880.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 7593. 21740. 33096. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 29895. 54190. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 54190.     0. ...     0.     0.     0.]\n",
      " [15348. 29895.  4959. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190. 54190. 54190. ...     0.     0.     0.]\n",
      " [21740.  4959.  4959. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [63124. 63124.  2703. ...  2703. 63124. 63124.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959. 33096.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 6210.  2703.  6210. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190. 54190.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [37880. 37880.     0. ...     0.     0.     0.]]\n",
      "x (128, 10) (128, 10) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 11102.     0. ...     0.     0.     0.]\n",
      " [ 2703.  6210.  2703. ...  2703.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [54190.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 6210.  6210.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210.  2703. ...  6210.  2703.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 4959. 21740.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [54190.  4959. 21740. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.  2703.  6210. ...     0.     0.     0.]\n",
      " [54190. 33096. 33096. ...  6210.  6210.  6210.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [21740.  2703.     0. ...     0.     0.     0.]\n",
      " [37880.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [15348.     0.     0. ...     0.     0.     0.]\n",
      " [21740. 29895.  4959. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 7593.  2703.  2703. ...     0.     0.     0.]\n",
      " [11102. 11102. 11102. ...     0.     0.     0.]\n",
      " [21740. 29895.  4959. ...     0.     0.     0.]]\n",
      "x (128, 15) (128, 15) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 6210.  6210.  2703. ...     0.     0.     0.]\n",
      " [ 2703. 21740. 54190. ...  2703.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210. 54190.  6210. ...     0.     0.     0.]\n",
      " [ 6210.  4959.  6210. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[54190. 33096. 54190. ...     0.     0.     0.]\n",
      " [ 2703. 37880.     0. ...     0.     0.     0.]\n",
      " [21740.  2703.  4959. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [63124.  2703. 63124. ...  2703.  2703.  2703.]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (128, 13) (128, 13) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[63124.  2703. 63124. ... 63124. 63124. 63124.]\n",
      " [33096. 33096. 21740. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [21740. 33096.     0. ...     0.     0.     0.]]\n",
      "x (128, 14) (128, 14) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 15348.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 9887.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703. 37880.     0. ...     0.     0.     0.]\n",
      " [ 4959. 15348. 21740. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.  2703.  4959. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  7593. ...  2703.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703. 37880. ...     0.     0.     0.]\n",
      " [ 6210. 54190. 29895. ...  2703. 21740.  4959.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...  2703.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [11102.  2703. 11102. ...     0.     0.     0.]]\n",
      "x (128, 10) (128, 10) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703. 29895.  2703. ...     0.     0.     0.]\n",
      " [ 6210.  6210.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[29895.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [37880.     0.     0. ...     0.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [ 2703.  7593.  7593. ...     0.     0.     0.]\n",
      " [11102. 63124.  2703. ...     0.     0.     0.]\n",
      " ...\n",
      " [54190.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]]\n",
      "x (128, 12) (128, 12) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[ 7593. 33096. 29895. ...     0.     0.     0.]\n",
      " [ 7593.  7593.  2703. ...  2703.  2703.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 2703.  2703.  2703. ...     0.     0.     0.]\n",
      " [ 6210.     0.     0. ...     0.     0.     0.]\n",
      " [11102.  4959.  4959. ... 11102.     0.     0.]]\n",
      "x (128, 11) (128, 11) (128, 10, 2048) (128, 10) (128, 10, 5) (128, 10, 8) (128, 1)\n",
      "[[21740.  2703. 29895. ...     0.     0.     0.]\n",
      " [ 2703.  2703.     0. ...     0.     0.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " ...\n",
      " [ 6210. 54190. 54190. ... 33096.  6210.     0.]\n",
      " [21740.     0.     0. ...     0.     0.     0.]\n",
      " [ 2703.     0.     0. ...     0.     0.     0.]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_D = data_generator(train_data)\n",
    "_i  = 0\n",
    "for d in train_D:\n",
    "    _i += 1\n",
    "    if  _i > 100:\n",
    "        break\n",
    "    print('x',d[0][0].shape, d[0][1].shape,d[0][2].shape, d[0][3].shape, d[0][4].shape, d[0][5].shape, d[1].shape)\n",
    "    print(d[0][5].sum(axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Evaluate(Callback):\n",
    "    def __init__(self, filename=None):\n",
    "        self.score = []\n",
    "        self.best = 0.\n",
    "        self.filename = filename\n",
    "       \n",
    "    def on_epoch_begin(self, epoch, logs=None):\n",
    "        if epoch ==  0:\n",
    "            print(\"[!] test load&save model\")\n",
    "            f = self.filename + \".h5\"\n",
    "            custom_objects = get_custom_objects()\n",
    "            self.model.save(f, include_optimizer=False, overwrite=True)\n",
    "            if \"bert\" in cfg[\"verbose\"]:\n",
    "                model_ = load_model(f, custom_objects=custom_objects)  \n",
    "            else:\n",
    "                model_ = load_model(f) \n",
    "    \n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "#         if epoch + 1 < 5:\n",
    "#             return\n",
    "        score = self.evaluate(self.model)\n",
    "        self.score.append((epoch, score))\n",
    "        \n",
    "        if epoch + 1 in SAVE_EPOCHS:\n",
    "            self.model.save(self.filename + \"_{}.h5\".format(epoch + 1), include_optimizer=False)             \n",
    "        if score > self.best:\n",
    "            self.model.save(self.filename + \".h5\", include_optimizer=False)\n",
    "            \n",
    "        if score > self.best:\n",
    "            self.best = score\n",
    "            print(\"[!] epoch = {}, new best score = {}\".format(epoch + 1,  score))\n",
    "        print('[!] epoch = {}, score = {}, best score: {}\\n'.format(epoch + 1, score, self.best))\n",
    "\n",
    "    def evaluate(self, model):\n",
    "        result = defaultdict(list)\n",
    "        for i in trange(len(val_data)):\n",
    "            d = val_data.iloc[i]\n",
    "            qid = d['query_id']\n",
    "            pid = d['product_id']\n",
    "            text = d['query']\n",
    "            label_words = d['label_words']\n",
    "            t1, t2 = token2id_X(text, x_dict=word_index, maxlen=cfg[\"maxlen\"])\n",
    "            \n",
    "            image = np.array(d['feature_convert'], dtype=\"float32\")\n",
    "            image = image[: cfg[\"max_box\"]]\n",
    "            img_mask = [1 for _ in image[: cfg[\"max_box\"]]]                   \n",
    "            pos = np.array(d['pos'], dtype=\"float32\")\n",
    "            pos = pos[: cfg[\"max_box\"]]\n",
    "            \n",
    "            image_char = [token2id_X(ent, x_dict=word_index)[0] for ent in label_words.split(IMAGE_LABEM_CONCAT_TOKEN)]\n",
    "            image_char = image_char[: cfg[\"max_box\"]]\n",
    "            image_char = pad_sequences(image_char, \n",
    "                                       maxlen=cfg[\"max_char\"], \n",
    "                                       dtype='int32',\n",
    "                                       padding='post',\n",
    "                                       truncating='post',\n",
    "                                       value=cfg[\"x_pad\"]) \n",
    "            output = model.predict([[t1], [t2], [image], [img_mask], [pos], [image_char]])\n",
    "            result[qid].append((pid, output[0][1]))\n",
    "            \n",
    "        query_id,product1,product2,product3,product4,product5 = [],[],[],[],[],[]\n",
    "        for key in result.keys():\n",
    "            rlist = result[key]\n",
    "            rlist.sort(key=lambda x: x[1], reverse=True)\n",
    "            query_id.append(key)\n",
    "            product1.append(rlist[0][0])\n",
    "            product2.append(rlist[1][0])\n",
    "            product3.append(rlist[2][0])\n",
    "            product4.append(rlist[3][0])\n",
    "            product5.append(rlist[4][0])\n",
    "        sub = pd.DataFrame({'query-id':query_id,\n",
    "                            'product1':product1,\n",
    "                            'product2':product2,\n",
    "                            'product3':product3,\n",
    "                            'product4':product4,\n",
    "                            'product5':product5,\n",
    "\n",
    "        })\n",
    "        sub.to_csv('../result/val_submission.csv',index=0)\n",
    "        \n",
    "        reference = json.load(open(VAL_ANS_PATH))\n",
    "        \n",
    "        # read predictions\n",
    "        k = 5\n",
    "        predictions = read_submission('../result/val_submission.csv', reference, k)\n",
    "\n",
    "        # compute score for each query\n",
    "        score_sum = 0.\n",
    "        for qid in reference.keys():\n",
    "            ground_truth_ids = set([str(pid) for pid in reference[qid]])\n",
    "            ref_vec = [1.0] * len(ground_truth_ids)\n",
    "            pred_vec = [1.0 if pid in ground_truth_ids else 0.0 for pid in predictions[qid]]\n",
    "            score_sum += get_ndcg(pred_vec, ref_vec, k)\n",
    "        # the higher score, the better\n",
    "        score = score_sum / len(reference)\n",
    "        \n",
    "        return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[!] fold_id = -1 starting\n",
      "{'verbose': '[image-bert-concat-query]-wwm_uncased_L12-768_v3_quart', 'base_dir': '/home/weiqiang/kdd2020/data/uncased_L-12_H-768_A-12/', 'maxlen': 20, 'max_box': 10, 'max_char': 8, 'lr': 0.0001, 'min_lr': 6e-08, 'opt': 'nadam', 'loss_w': 20.0, 'trainable': True, 'bert_trainable': True, 'mix_mode': '', 'unit1_1': 128, 'accum_step': 1, 'cls_num': 4, 'raw_filename': '{}_{}oof{}', 'x_pad': 0, 'filename': '[image-bert-concat-query]-wwm_uncased_L12-768_v3_quart_20oof-1', 'num_example': 10000}\n",
      "/home/weiqiang/kdd2020/data/uncased_L-12_H-768_A-12/bert_config.json /home/weiqiang/kdd2020/data/uncased_L-12_H-768_A-12/bert_model.ckpt\n",
      "WARNING:tensorflow:From /home/weiqiang/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/weiqiang/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "image_char_input (InputLayer)   (None, None, 8)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "CharEmbedding (TimeDistributed) (None, None, 128)    18544928    image_char_input[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "image_input (InputLayer)        (None, None, 2048)   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, None, 2176)   0           CharEmbedding[0][0]              \n",
      "                                                                 image_input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "image_pos_input (InputLayer)    (None, None, 5)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "query_token_input (InputLayer)  (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "image_embed (Dense)             (None, None, 512)    1114624     concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "image_mask_input (InputLayer)   (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "pos_embed (Dense)               (None, None, 512)    3072        image_pos_input[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "embed_layer (Embedding)         (None, None, 600)    18314400    query_token_input[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "token_mask (Lambda)             (None, None)         0           query_token_input[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, None, 512)    0           image_embed[0][0]                \n",
      "                                                                 image_mask_input[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, None, 512)    0           pos_embed[0][0]                  \n",
      "                                                                 image_mask_input[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, None, 600)    0           embed_layer[0][0]                \n",
      "                                                                 token_mask[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, None, 512)    0           lambda_4[0][0]                   \n",
      "                                                                 lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "query_segm_input (InputLayer)   (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, None, 128)    747520      lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, None, 1152)   15353856    add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "model_3 (Model)                 multiple             103788288   query_token_input[0][0]          \n",
      "                                                                 query_segm_input[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, None, 128)    0           bidirectional_1[0][0]            \n",
      "                                                                 token_mask[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, None, 1152)   0           bidirectional_2[0][0]            \n",
      "                                                                 image_mask_input[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, 896)    0           model_3[1][0]                    \n",
      "                                                                 lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 1152)         0           lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 896)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 2048)         0           lambda_7[0][0]                   \n",
      "                                                                 lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 2048)         4196352     concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 512)          1049088     dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 128)          65664       dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 2)            258         dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 144,863,650\n",
      "Trainable params: 144,863,650\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "WARNING:tensorflow:From /home/weiqiang/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/weiqiang/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Epoch 1/20\n",
      "[!] test load&save model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/weiqiang/anaconda3/lib/python3.7/site-packages/keras/engine/saving.py:292: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "  warnings.warn('No training configuration found in save file: '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79/79 [==============================] - 93s 1s/step - loss: 0.6302 - acc: 0.6269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14720/14720 [04:46<00:00, 51.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] epoch = 1, new best score = 0.23371212308222292\n",
      "[!] epoch = 1, score = 0.23371212308222292, best score: 0.23371212308222292\n",
      "\n",
      "Epoch 2/20\n",
      "79/79 [==============================] - 24s 302ms/step - loss: 0.4874 - acc: 0.7646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14720/14720 [04:41<00:00, 52.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] epoch = 2, new best score = 0.303755431865909\n",
      "[!] epoch = 2, score = 0.303755431865909, best score: 0.303755431865909\n",
      "\n",
      "Epoch 3/20\n",
      "35/79 [============>.................] - ETA: 13s - loss: 0.4461 - acc: 0.7931"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-e631eabb1ba3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     22\u001b[0m                           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_EPOCH\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m                           \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m                           \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m                           )\n\u001b[1;32m     26\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\\n[!] fold_id = {} finish\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfold_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1420\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m    215\u001b[0m                 outs = model.train_on_batch(x, y,\n\u001b[1;32m    216\u001b[0m                                             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m                                             class_weight=class_weight)\n\u001b[0m\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1215\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1216\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1217\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1218\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "gc.collect()\n",
    "fold_id = -1\n",
    "print(\"\\n\\n[!] fold_id = {} starting\".format(fold_id))\n",
    "cfg[\"filename\"] = cfg[\"raw_filename\"].format(cfg[\"verbose\"], FOLD_NUM, fold_id)\n",
    "cfg[\"num_example\"] = len(train_data)\n",
    "\n",
    "K.clear_session()\n",
    "gc.collect()\n",
    "seed(SEED - fold_id)\n",
    "np.random.seed(SEED - fold_id)\n",
    "tf.random.set_random_seed(SEED - fold_id)\n",
    "train_D = data_generator(train_data)\n",
    "print(cfg)\n",
    "model = build_model(cfg, summary=True, \n",
    "                    word_embedding_matrix=word_embedding_matrix,\n",
    "                    )\n",
    "#     model.load_weights('[image-concat-query]-wwm_uncased_L24-1024_20oof0')\n",
    "evaluator = Evaluate(filename=cfg[\"filename\"])\n",
    "#     checkpoint= ModelCheckpoint(cfg[\"filename\"], monitor='acc', verbose=1, save_best_only=False, mode='max') \n",
    "model.fit_generator(train_D.__iter__(),\n",
    "                          steps_per_epoch=len(train_D),\n",
    "                          epochs=MAX_EPOCH,\n",
    "                          callbacks=[evaluator],\n",
    "                          shuffle=True\n",
    "                          )\n",
    "print(\"\\n\\n[!] fold_id = {} finish\".format(fold_id))\n",
    "del model, evaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish\n"
     ]
    }
   ],
   "source": [
    "with open('../data/test_data.pkl', 'rb') as outp:\n",
    "    test_data = pickle.load(outp)\n",
    "\n",
    "f = cfg[\"filename\"] + \".h5\"\n",
    "if \"bert\" in cfg[\"verbose\"]:\n",
    "    custom_objects = get_custom_objects()\n",
    "    model = load_model(f, custom_objects=custom_objects)  \n",
    "else:\n",
    "    model = load_model(f)\n",
    "print(\"finish\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28830/28830 [09:27<00:00, 50.77it/s]\n"
     ]
    }
   ],
   "source": [
    "gc.collect()\n",
    "result = defaultdict(list)\n",
    "for i in trange(len(test_data)):\n",
    "    d = test_data.iloc[i]\n",
    "    qid = d['query_id']\n",
    "    pid = d['product_id']\n",
    "    text = d['query']\n",
    "    label_words = d['label_words']\n",
    "    t1, t2 = token2id_X(text, x_dict=word_index, maxlen=cfg[\"maxlen\"])\n",
    "\n",
    "    image = np.array(d['feature_convert'], dtype=\"float32\")\n",
    "    image = image[: cfg[\"max_box\"]]\n",
    "    img_mask = [1 for _ in image[: cfg[\"max_box\"]]]                   \n",
    "    pos = np.array(d['pos'], dtype=\"float32\")\n",
    "    pos = pos[: cfg[\"max_box\"]]\n",
    "\n",
    "    image_char = [token2id_X(ent, x_dict=word_index)[0] for ent in label_words.split(IMAGE_LABEM_CONCAT_TOKEN)]\n",
    "    image_char = image_char[: cfg[\"max_box\"]]\n",
    "    image_char = pad_sequences(image_char, \n",
    "                               maxlen=cfg[\"max_char\"], \n",
    "                               dtype='int32',\n",
    "                               padding='post',\n",
    "                               truncating='post',\n",
    "                               value=cfg[\"x_pad\"]) \n",
    "    output = model.predict([[t1], [t2], [image], [img_mask], [pos], [image_char]])\n",
    "    result[qid].append((pid, output[0][1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_id,product1,product2,product3,product4,product5 = [],[],[],[],[],[]\n",
    "for key in result.keys():\n",
    "    rlist = result[key]\n",
    "    rlist.sort(key=lambda x: x[1], reverse=True)\n",
    "    query_id.append(key)\n",
    "    product1.append(rlist[0][0])\n",
    "    product2.append(rlist[1][0])\n",
    "    product3.append(rlist[2][0])\n",
    "    product4.append(rlist[3][0])\n",
    "    product5.append(rlist[4][0])\n",
    "\n",
    "sub = pd.DataFrame({'query-id':query_id,\n",
    "                    'product1':product1,\n",
    "                    'product2':product2,\n",
    "                    'product3':product3,\n",
    "                    'product4':product4,\n",
    "                    'product5':product5,\n",
    "\n",
    "})\n",
    "\n",
    "sub.to_csv('../result/submission.csv',index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query-id</th>\n",
       "      <th>product1</th>\n",
       "      <th>product2</th>\n",
       "      <th>product3</th>\n",
       "      <th>product4</th>\n",
       "      <th>product5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>103032634</td>\n",
       "      <td>103053460</td>\n",
       "      <td>103006984</td>\n",
       "      <td>103036197</td>\n",
       "      <td>103035030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>103047501</td>\n",
       "      <td>103058164</td>\n",
       "      <td>103050362</td>\n",
       "      <td>103037741</td>\n",
       "      <td>103000844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>103040094</td>\n",
       "      <td>103047101</td>\n",
       "      <td>103045222</td>\n",
       "      <td>103003414</td>\n",
       "      <td>103054005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>103020362</td>\n",
       "      <td>103064876</td>\n",
       "      <td>103007742</td>\n",
       "      <td>103032254</td>\n",
       "      <td>103031371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>103061976</td>\n",
       "      <td>103032810</td>\n",
       "      <td>103064801</td>\n",
       "      <td>103054847</td>\n",
       "      <td>103022805</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query-id   product1   product2   product3   product4   product5\n",
       "0         0  103032634  103053460  103006984  103036197  103035030\n",
       "1         1  103047501  103058164  103050362  103037741  103000844\n",
       "2         2  103040094  103047101  103045222  103003414  103054005\n",
       "3         3  103020362  103064876  103007742  103032254  103031371\n",
       "4         4  103061976  103032810  103064801  103054847  103022805"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
